{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGBoost Parameter Tuning for Rental Listing Inquiries\n",
    "## 刘晓明 第三周作业（part 2-1）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 第二步：调整树的参数：max_depth & min_child_weight\n",
    "\n",
    "(粗调，参数的步长为2；下一步是在粗调最佳参数周围，将步长降为1，进行精细调整)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "首先 import 必要的模块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# path to where the data lies\n",
    "dpath = './data/'\n",
    "train = pd.read_csv(dpath +\"RentListingInquries_FE_train.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop ids and get labels\n",
    "y_train = train['interest_level']\n",
    "\n",
    "train = train.drop([\"interest_level\"], axis=1)\n",
    "X_train = np.array(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "各类样本不均衡，交叉验证是采用StratifiedKFold，在每折采样时各类样本按比例采样"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "第一轮参数调整得到的n_estimators最优值（220），其余参数继续默认值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "用交叉验证评价模型性能时，用scoring参数定义评价指标。评价指标是越高越好，因此用一些损失函数当评价指标时，需要再加负号，如neg_log_loss，neg_mean_squared_error 详见sklearn文档：http://scikit-learn.org/stable/modules/model_evaluation.html#log-loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': range(3, 10, 2), 'min_child_weight': range(1, 6, 2)}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "max_depth = range(3,10,2)\n",
    "min_child_weight = range(1,6,2)\n",
    "param_test2_1 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.60221, std: 0.00318, params: {'max_depth': 3, 'min_child_weight': 1},\n",
       "  mean: -0.60259, std: 0.00328, params: {'max_depth': 3, 'min_child_weight': 3},\n",
       "  mean: -0.60249, std: 0.00324, params: {'max_depth': 3, 'min_child_weight': 5},\n",
       "  mean: -0.58874, std: 0.00413, params: {'max_depth': 5, 'min_child_weight': 1},\n",
       "  mean: -0.58963, std: 0.00368, params: {'max_depth': 5, 'min_child_weight': 3},\n",
       "  mean: -0.58945, std: 0.00342, params: {'max_depth': 5, 'min_child_weight': 5},\n",
       "  mean: -0.58914, std: 0.00317, params: {'max_depth': 7, 'min_child_weight': 1},\n",
       "  mean: -0.58937, std: 0.00370, params: {'max_depth': 7, 'min_child_weight': 3},\n",
       "  mean: -0.58880, std: 0.00357, params: {'max_depth': 7, 'min_child_weight': 5},\n",
       "  mean: -0.60226, std: 0.00388, params: {'max_depth': 9, 'min_child_weight': 1},\n",
       "  mean: -0.59922, std: 0.00566, params: {'max_depth': 9, 'min_child_weight': 3},\n",
       "  mean: -0.59578, std: 0.00374, params: {'max_depth': 9, 'min_child_weight': 5}],\n",
       " {'max_depth': 5, 'min_child_weight': 1},\n",
       " -0.5887351903284485)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=220,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_test2_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_1.grid_scores_, gsearch2_1.best_params_,     gsearch2_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([ 2.23074861,  2.65763454,  2.69463925,  3.30102944,  3.62995882,\n",
       "         3.59766273,  4.28756704,  4.49844666,  4.48730793,  5.3142704 ,\n",
       "         5.55349984,  4.68907838]),\n",
       " 'mean_score_time': array([ 0.16698532,  0.07552595,  0.10664678,  0.10792656,  0.0900722 ,\n",
       "         0.11578994,  0.12307835,  0.12351956,  0.13793001,  0.17913656,\n",
       "         0.16294136,  0.10563679]),\n",
       " 'mean_test_score': array([-1.89738453, -1.89749741, -1.89794523, -1.84625175, -1.84746907,\n",
       "        -1.84842319, -1.81425999, -1.81819681, -1.82191243, -1.80225298,\n",
       "        -1.80935193, -1.81214152]),\n",
       " 'mean_train_score': array([-1.89582817, -1.89601021, -1.89646037, -1.84306682, -1.84496196,\n",
       "        -1.84615697, -1.80789678, -1.81299341, -1.81775794, -1.79172928,\n",
       "        -1.80148929, -1.80581986]),\n",
       " 'param_max_depth': masked_array(data = [3 3 3 5 5 5 7 7 7 9 9 9],\n",
       "              mask = [False False False False False False False False False False False False],\n",
       "        fill_value = ?),\n",
       " 'param_min_child_weight': masked_array(data = [1 3 5 1 3 5 1 3 5 1 3 5],\n",
       "              mask = [False False False False False False False False False False False False],\n",
       "        fill_value = ?),\n",
       " 'params': ({'max_depth': 3, 'min_child_weight': 1},\n",
       "  {'max_depth': 3, 'min_child_weight': 3},\n",
       "  {'max_depth': 3, 'min_child_weight': 5},\n",
       "  {'max_depth': 5, 'min_child_weight': 1},\n",
       "  {'max_depth': 5, 'min_child_weight': 3},\n",
       "  {'max_depth': 5, 'min_child_weight': 5},\n",
       "  {'max_depth': 7, 'min_child_weight': 1},\n",
       "  {'max_depth': 7, 'min_child_weight': 3},\n",
       "  {'max_depth': 7, 'min_child_weight': 5},\n",
       "  {'max_depth': 9, 'min_child_weight': 1},\n",
       "  {'max_depth': 9, 'min_child_weight': 3},\n",
       "  {'max_depth': 9, 'min_child_weight': 5}),\n",
       " 'rank_test_score': array([10, 11, 12,  7,  8,  9,  4,  5,  6,  1,  2,  3], dtype=int32),\n",
       " 'split0_test_score': array([-1.904837  , -1.9048393 , -1.90537758, -1.86813661, -1.86922186,\n",
       "        -1.87028639, -1.80864564, -1.81205344, -1.81534182, -1.7955135 ,\n",
       "        -1.80454465, -1.80950213]),\n",
       " 'split0_train_score': array([-1.90079462, -1.90087334, -1.90126588, -1.86396653, -1.86575001,\n",
       "        -1.86690596, -1.80110119, -1.8061502 , -1.80999227, -1.78507966,\n",
       "        -1.79747592, -1.80407026]),\n",
       " 'split1_test_score': array([-1.89414433, -1.89418724, -1.89449427, -1.84260162, -1.84371216,\n",
       "        -1.84471012, -1.81933539, -1.82307976, -1.82619675, -1.79917828,\n",
       "        -1.80483572, -1.80860676]),\n",
       " 'split1_train_score': array([-1.89538117, -1.89557858, -1.89589882, -1.84090216, -1.84306947,\n",
       "        -1.84431668, -1.81445103, -1.8197674 , -1.82399422, -1.78883   ,\n",
       "        -1.79693318, -1.80241998]),\n",
       " 'split2_test_score': array([-1.90097688, -1.90122587, -1.90157714, -1.84294482, -1.8439429 ,\n",
       "        -1.84512163, -1.81001654, -1.81433944, -1.81827517, -1.80576014,\n",
       "        -1.81140141, -1.81747666]),\n",
       " 'split2_train_score': array([-1.89983001, -1.90005227, -1.9005272 , -1.83932917, -1.84119965,\n",
       "        -1.84252076, -1.80457729, -1.80961677, -1.81502997, -1.79441037,\n",
       "        -1.80353913, -1.81113164]),\n",
       " 'split3_test_score': array([-1.90253915, -1.90250469, -1.90300397, -1.8420231 , -1.84330308,\n",
       "        -1.84418303, -1.80879493, -1.81282132, -1.81611468, -1.80624298,\n",
       "        -1.8142515 , -1.81280824]),\n",
       " 'split3_train_score': array([-1.90115811, -1.90137115, -1.90198618, -1.84171322, -1.84326367,\n",
       "        -1.84447145, -1.80390443, -1.808715  , -1.81286745, -1.79601218,\n",
       "        -1.8060183 , -1.80577938]),\n",
       " 'split4_test_score': array([-1.8844191 , -1.88472388, -1.88526713, -1.83554503, -1.83715799,\n",
       "        -1.83780728, -1.82451212, -1.82869492, -1.8336392 , -1.80457228,\n",
       "        -1.81172845, -1.81231459]),\n",
       " 'split4_train_score': array([-1.88197695, -1.88217572, -1.88262378, -1.82942303, -1.83152702,\n",
       "        -1.83257   , -1.81544997, -1.82071769, -1.82690579, -1.79431418,\n",
       "        -1.80347994, -1.80569803]),\n",
       " 'std_fit_time': array([ 0.4208097 ,  0.19117497,  0.1765739 ,  0.1672316 ,  0.11324032,\n",
       "         0.15489323,  0.13125475,  0.15735095,  0.17904223,  0.13649619,\n",
       "         0.19436801,  0.38385669]),\n",
       " 'std_score_time': array([ 0.03265332,  0.01407311,  0.04169902,  0.04168524,  0.02764364,\n",
       "         0.02179954,  0.0489764 ,  0.03210032,  0.02548046,  0.03983879,\n",
       "         0.0646084 ,  0.05419839]),\n",
       " 'std_test_score': array([ 0.00739684,  0.00730685,  0.00730231,  0.0112763 ,  0.01116656,\n",
       "         0.01125537,  0.00648432,  0.00656567,  0.00701213,  0.00420672,\n",
       "         0.00393359,  0.00311136]),\n",
       " 'std_train_score': array([ 0.00722696,  0.00721537,  0.00723786,  0.01134406,  0.01125399,\n",
       "         0.01126567,  0.00588467,  0.00603469,  0.00654582,  0.00411914,\n",
       "         0.00362054,  0.00292752])}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.588735 using {'max_depth': 5, 'min_child_weight': 1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAELCAYAAADz6wBxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8lPW5///XNUv2sAaQRUVRKsgSIQRUgqAEQa2txeP21RYreqrHY+v5adXWU6tHWz2eY9VWPbW41bpVWz3Yg5rgBq4QEFxABRU1gmwKZCHJTOb6/XHfmdwJk2TIZDJZrufjMQ9m7vtz33NlQu73fO7tI6qKMcYY016+VBdgjDGme7MgMcYYkxALEmOMMQmxIDHGGJMQCxJjjDEJsSAxxhiTEAsSY4wxCbEgMcYYkxALEmOMMQkJpLqAzpCXl6cjR45MdRnGGNOtrFq1aoeqDmqrXa8IkpEjR1JWVpbqMowxplsRkc/jaWe7towxxiTEgsQYY0xCLEiMMcYkxILEGGNMQixIjDHGJMSCxBhjTEIsSIwxxiSkV1xHYkxniUSUmnA9NaEINaF69xFxp9VT2zDdbROujyAi+H2CT8An4jx8jc8b5okI/mbzvG33abfPeprOE8Fdplk7EcRH7Hbu+o3xsiAxPZaqEqrXfTfing17kw1+OEJtqNn0JqEQobb5cp75taEIdfWRVP/YSSdu4PmbhUzTwGkMRr8vRrto0HkC1H0eO+iaBW0c7xtdX/P39bXQzn3eKfX54v854v38/M1ql04MfQsS02kiEaU2HGsD3bhxr42xcfduuGtb/LbvCYFw4/yItq9Wn0BG0O88Aj4ygn7Sg34ygj4yAn7ycgKN84M+0gONz73LZAT9pEeXb5jX2C7o96GqRBTqVYlEFG147r6OKERUqW82T1Wpj7Bvu2bzmrSLrpcW2jWuJ9JQVyRGO3ee877N2rmvm6zP267Ze8V8X7e+UH2kaTt3nre+hnU3zGupvibtmr1vT+UTKLn8OA4bnJPU97Eg6aVa+7beZGPdbMNdG262EW+24a5t5dt+Xbj939bT/L7GDbG7MW94npMeYGC2ZyPebH40BDwb94wYG3Xvhj7oF9uF04s0CRw3pFoNOm8wtRF0jUHcrF2ygjjStN2A7LSkf34WJF1ET/q2PiA7rckGOrqRbv6t3TMt1rd177d9v8826iZ5fD7Bh/0fay8LklZ89dtbqPmyHJl6NPUFU6ntl9fK/nP7tm6M6Z2SGiQiMhe4A/ADi1T15hhtzgB+DSiwVlXPcaf/CLjWbXajqj4kIlnAk8AooB54VlWvTlb9z328k++8s5K8l5YC8HmfoZQNOYKyIUewbsBI6n3+fZYRYd9v44HGjbh9WzfG9DSimpwjTSLiBz4GioFyYCVwtqqu87Q5HPgrcLyqfisig1V1m4gMAMqAApyAWQVMBmqBqar6soikAS8Cv1HV51qrpaCgQNtzG/nSdVv5evde+n79Bf3fKyPn3ZWkr38Pqa9Hs7LxFRSSdsyxZE4vImv4UDKCPtL8Pvu2bozpEURklaoWtNUumT2SQmCjqn7qFvQ48D1gnafNhcBdqvotgKpuc6efCJSq6jfusqXAXFV9DHjZbVsnIquBEcn6AYrHDnGfjYTTZgBQX1lJ1ZtvUrVsOZXLl1O77GVqgb2jR5Mzo4jsohlkTToKCQaTVZYxxnQpyQyS4cCXntflwNRmbUYDiMjrOLu/fq2qz7ew7HDvgiLSD/guzq6zTuPPyaFPcTF9iotRVWo3bKBq+XIqly1n50N/Zuei+/BlZ5N9zNFkFxWRM2MGwQMO6MwSjTGmUyUzSGLt32m+Hy0AHA7MxOlZLBeRcW0tKyIB4DHgzoYezz5vLnIRcBHAQQcdtL+1x0VEyBg9mozRoxl4wQXUV1ZR/dabVLq9lYpS59hK+uGHkz2jiJyG3kpa8k/HM8aYzpLMICkHDvS8HgFsjtHmLVUNAZ+JyEc4wVKOEy7eZV/xvL4X2KCqt7f05qp6r9uOgoKCTrnkyJ+TTe7s2eTOno2qUrdxYzRUvvnzw3xz3/34srLIOuZocopmkDOjiODQoZ1RmjHGJE0yD7YHcA62nwB8hXOw/RxV/cDTZi7OAfgfiUge8A6QT+MB9klu09XAZFX9RkRuBMYA/6SqcZ0z296D7R2pvrKK6rffcoNlGeHNWwBIP/wwst1QyZo0yXorxpguI96D7UkLEreIk4DbcY5/3K+qN4nIDUCZqi4W5/Sm/wbm4pzOe5OqPu4u+2PgF+6qblLVB0RkBM6xkw9xzuAC+IOqLmqtjq4QJF6qSt0nn0RDpbpsFYRCTm/l6KPJKSpyeivDhqW6VGNML9YlgqSr6GpB0lykqoqqt9+mctkyqpYtJ7TZ2QOYdtio6C6wzMmT8VlvxRjTiSxIPLp6kHipKnWffkrlsuVULV9G9coyNBRCsrLInjaNnBlF5BQVERw+vO2VGWNMAixIPLpTkDTn9FZWULnc7a189RUAaaNGRXeBZRYUWG/FGNPhLEg8unOQeKkqdZ99Ft0FVr1yZWNvZerU6AWRaSOst2KMSZwFiUdPCZLmItXVVL39dvSCyFB5OQBphx5KTlER2TOKyJoyxXorxph2sSDx6KlB4uX0VjZRtXwZlQ29lbo6JDOT7KlTnQsiZ8wgbUTS7ihjjOlhLEg8ekOQNBfZu9fprbgXRIa+dO44kzZyZONV9oVT8KWnp7hSY0xXZUHi0RuDxEtVqdu0KboLrHrFCqe3kpFB1tTC6CnGaUm6lYwxpnuyIPHo7UHSXGTvXqpXrIjeviX0xRcApB18MNkzZpAzw3orxhgLkiYsSFpXt2lTNFSqV6xAa2utt2KMsSDxsiCJX6SmxtNbWUboc6e3Ejz4oGioZBUW4svISHGlxphksyDxsCBpv7rPP2+8J9jbbm8lPZ2swsLoBZFpI0emukxjTBJYkHhYkHSMSE0N1StXOrdvWbaMus8/ByB40EHRUMkqLMSXmZniSo0xHcGCxMOCJDnqvviiaW+lpsbprUyZ4l5l7/RWbAx7Y7onCxIPC5Lki9TWUr1iZfSeYHWbNgEQPPDA6FX22VOnWm/FmG7EgsTDgqTz1X35ZfSeYFVvv+30VtLSPL2VGaQdYr0VY7oyCxIPC5LUitTWUr2yLHr7lrrPPgMgOGJEdBdY9tSp+LKyUlypMcbLgsTDgqRrqSsvj15lX/XWW+jevU5vpaCg8Z5ghxxivRVjUsyCxMOCpOuK1NWxt6wsekFk3SefABAcPjx6T7DsadZbMSYVLEg8LEi6j7ryr6h6zdNbqa5GgkGyphSQ3XCV/aGHWm/FmE5gQeJhQdI9tdhbGTYsugsse+pUfNnZKa7UmJ6pSwSJiMwF7gD8wCJVvTlGmzOAXwMKrFXVc9zpPwKudZvdqKoPudMnAw8CmcAS4Kfaxg9hQdIzhL76isrlr1G5fDlVb74Z7a1kFkxuvCfYqFHWWzGmg6Q8SETED3wMFAPlwErgbFVd52lzOPBX4HhV/VZEBqvqNhEZAJQBBTgBswqY7LZZAfwUeAsnSO5U1edaq8WCpOfRujqqV692doEtX0btho0ABIYNjYZK9rRp1lsxJgHxBkkgiTUUAhtV9VO3oMeB7wHrPG0uBO5S1W8BVHWbO/1EoFRVv3GXLQXmisgrQB9VfdOd/mfg+0CrQWJ6HklLI3vaNLKnTYOfX0lo82a3t7KMPc8+y64nnoBgkKzJkxvvCXbYYdZbMSYJkhkkw4EvPa/LganN2owGEJHXcXZ//VpVn29h2eHuozzG9H2IyEXARQAH2S3Qe7zgsGH0P/MM+p95httbeSd6lf22W29l2623Ehg6tPGeYNOOxp9jvRVjOkIygyTWV7/m+9ECwOHATGAEsFxExrWybDzrdCaq3gvcC86urfhKNj2B01uZSva0qXDllYS2bHGOqyxfzp7/+z92/fWvTm9l0qToBZHphx9uvRVj2imZQVIOHOh5PQLYHKPNW6oaAj4TkY9wgqUcJ1y8y77iTh/RxjqNaSI4dCj9zziD/me4vZV31kSvst9263/Brf9F4IADGu8JdvTR+HNyUl22Md1GMg+2B3AOtp8AfIVzsP0cVf3A02YuzgH4H4lIHvAOkE/jAfZJbtPVOAfbvxGRlcC/Am/jHGz/vaouaa0WO9huWhL6+munt7JsOVVvvEGkqgoCAU9vZQbpo623YnqnlJ+15RZxEnA7zvGP+1X1JhG5AShT1cXi/HX+NzAXqAduUtXH3WV/DPzCXdVNqvqAO72AxtN/nwP+1U7/NR1BQyGq33knevuW2o8+AnB7K9Ode4Idc4z1Vkyv0SWCpKuwIDHtEdq6tfGeYG+8QaSy0umtHHVU9ILI9NGjrbdieiwLEg8LEpMoDYXYu2ZN9Cr72g8/BCAwZAjZRdOde4IdczT+3NwUV2pMx7Eg8bAgMR0ttHVb4z3B3niDSEWF01vJzyd7hnNBZPp3vmO9FdOtWZB4WJCYZNJQiL1r1zb2VtavByAweLDbW3GPrfTpk+JKjdk/FiQeFiSmM4W2baOq4Z5gr7/u9Fb8fjLz86MXRKaPGWO9FdPlWZB4WJCYVNFw2NNbWUbtOqe34h+UR870InKOO46coul2TzDTJVmQeFiQmK4itG0bVa+97ty+5fU3iOzZg6Snk100nT7FxeTMmmW7wEyXYUHiYUFiuiINh6letZqK0lIqSksJb90KwSDZ06aRWzyb3NmzCQwYkOoyTS9mQeJhQWK6Oo1EqHn3XfaUlFJRUkKovBx8PrIKCsgtLiZ3TjHBIUNSXabpZSxIPCxITHeiqtR++CF7SkqoKC2lbqMzMmTmxInkzplD7pxi0g48sI21GJM4CxIPCxLTndV++ikVJSVUlJRSs84Zzid9zBj6zCkmt7iY9MMOS3GFpqeyIPGwIDE9RV15ORUlzjGVve+8A0DaoYeS64ZKxtixdlqx6TAWJB4WJKYnCm3dRsXSUipKSqleuRIiEYIjRkSPqWROnIj4fKku03RjFiQeFiSmpwt/+y2VL77InpISqt58C0IhAoMHkzt7Nrlz5pBVMBkJJHP4IdMTWZB4WJCY3qS+ooLKV16hoqSEyuWvoTU1+Pv3J+eE4+kzZw7Z06YhaWmpLtN0AxYkHhYkpreKVFdTufw1J1ReeYVIVRW+nBxyZs0id04xOdOn48vMTHWZpouyIPGwIDEGInV1VL3xBhUlpVS++CL1u3cjmZnkFBWRO2cOOTOPs0G7TBMWJB4WJMY0paEQ1WVlzrUqS5dSv30HEgySfcwxTqgcP4tA//6pLtOkmAWJhwWJMS3TSIS9a9ZQ8UIJe0pLCG/eAn4/2VMLnTPAZs8mMGhQqss0KWBB4mFBYkx8VJWaD9a5F0CWULdpE4iQOWkSucWz6VNcTHD48FSXaTpJlwgSEZkL3AH4gUWqenOz+QuAW4Gv3El/UNVF7rxbgJPd6f+hqk+4009wl/EBlcACVd3YWh0WJMbsP1WldsMG56aSJaXUfvQRABnjxkWvVUk/5JAUV2mSKeVBIiJ+4GOgGCgHVgJnq+o6T5sFQIGqXtps2ZOBnwHzgHTgVeB4Vd0jIh8D31PV9SJyCVCoqgtaq8WCxJjE1X3+ORWlpewpKaXm3XcBSD/8cCdUTpxD+ujRdlV9DxNvkCTzCqVCYKOqfuoW9DjwPWBdq0s5xgKvqmoYCIvIWmAu8FdAgYYBG/oCmzu6cGPMvtIOPpiBCxcycOFCQlu2UFG6lIqSEnbccw877r6b4MEH0WfOHOdWLePHW6j0IskMkuHAl57X5cDUGO3mi8gMnN7L5ar6JbAWuE5EbgOygFk0BtBCYImI7AX2ANOSVL8xpgXBoUMZ8MPzGPDD8wjv2EHF0hepKC1l5wMPsvNPiwgMHRo9ppI5aRLi96e6ZJNEyQySWF9Hmu9HexZ4TFVrReQnwEM4u7BKRGQK8AawHXgTCLvLXA6cpKpvi8iVwG044dL0zUUuAi4COOigg/YpJBQKUV5eTk1NTbt+ONO9ZGRkMGLECILBYKpL6XECeXn0P+tM+p91JvW7dlHx8itUlJay6/En+PbPD+PPyyP3hBPInVNMdmEhYr+DHieZx0iOBn6tqie6r68BUNXfttDeD3yjqn1jzHsU+AvOcZa3VHWUO/0g4HlVHdtaLbGOkXz22Wfk5uYycOBA64L3cKrKzp07qaio4BA7ONxp6iurqFr2KntKSqlctgytrsbXty+5s2aRO2cO2ccegy89PdVlmlZ0hWMkK4HDReQQnLOyzgLO8TYQkaGqusV9eSqw3p3uB/qp6k4RmQBMAErcdn1FZLSqNhzIX9+e4mpqahg5cqSFSC8gIgwcOJDt27enupRexZ+TTZ+TTqLPSScRqamh6vXXndOKX3yR3c88gy8ri5yZxzkXQBYV4cvOTnXJpp2SFiSqGhaRS4EXcE7/vV9VPxCRG4AyVV0MXCYip+LstvoGWOAuHgSWuxv5PcC57oF3RORC4G8iEgG+BX7c3hotRHoP+12nli8jw9m9dcIJaF0dVW+viIbKniXPIenpZE+fTp85xeTMnIm/7z47JkwX1msvSFy/fj1jxoxJUUUmFex33vVofT3Vq1ZFB+sKb90KgQDZ06Y5g3XNnk1gwIBUl9lrxbtry0a9SZFdu3Zx9913t2vZ22+/nerq6g6uqGPMnDmT9l6z88wzz7BuXePZ4fGsq6amhsLCQiZOnMiRRx7Jdddd1673Nqkhfj/ZhYUccO0vOezllxj5xOMM+NEPqfv8c77+1XVsmF7E5+f9kG8e/guhrVtTXa5pgQVJivTUIElE8yCJR3p6Oi+99BJr165lzZo1PP/887z11ltJqtAkk/h8ZE6cyJArr2RUyQsc8szT5P3knwl/+w1bb7qJjcfNZNOZZ7Hzvvuo+/LLtldoOo0NmQZc/+wHrNu8p0PXOXZYH6777pEtzr/66qv55JNPyM/Pp7i4mMGDB/PXv/6V2tpaTjvtNK6//nqqqqo444wzKC8vp76+nn//939n69atbN68mVmzZpGXl8fLL78cc/05OTn8y7/8C0uXLqV///785je/4ec//zlffPEFt99+O6eeeiqbNm3ivPPOo6qqCoA//OEPHHPMMTz99NPcddddlJaW8vXXX3PcccexbNkyDjjggH3eZ+/evZx//vmsW7eOMWPGsHfv3ui8kpISrrvuOmpraxk1ahQPPPAAOTk5jBw5kjPPPDNa+6OPPsq2bdtYvHgxr776KjfeeCN/+9vfAHjyySe55JJL2LVrF/fddx9FRUVN3l9EyHFvfR4KhQiFQnY8pAcQETKOOIKMI45g0GWXUfvpp87ur5IStt36X2y79b9IHzPGuVZlzhzSDzss1SX3atYjSZGbb76ZUaNGsWbNGoqLi9mwYQMrVqxgzZo1rFq1imXLlvH8888zbNgw1q5dy/vvv8/cuXO57LLLGDZsGC+//HKLIQJQVVXFzJkzWbVqFbm5uVx77bWUlpby9NNP86tf/QqAwYMHU1payurVq3niiSe47LLLADjttNM44IADuOuuu7jwwgu5/vrrY4YIwD333ENWVhbvvvsuv/zlL1m1ahUAO3bs4MYbb2Tp0qWsXr2agoICbrvttuhyffr0YcWKFVx66aX87Gc/45hjjuHUU0/l1ltvZc2aNYwaNQqAcDjMihUruP3227n++usB2Lx5MyeddFJ0XfX19eTn5zN48GCKi4uZOjXWda+mO0s/9FDyfvLPHPL3vzFq6VIGX3UVvowMdtz5ez495bt8ctLJbPvd7ez94AN6w3HfrsZ6JNBqz6EzlJSUUFJSwlFHHQVAZWUlGzZsoKioiCuuuIKrrrqKU045ZZ9v461JS0tj7ty5AIwfP5709HSCwSDjx49n06ZNgPMN/tJLL2XNmjX4/X4+/vjj6PK///3vGTduHNOmTePss89u8X2WLVsWDaAJEyYwYcIEAN566y3WrVvHscceC0BdXR1HH310dLmGdZ599tlcfvnlLa7/Bz/4AQCTJ0+O1j1s2DCWLFkSbeP3+1mzZg27du3itNNO4/3332fcuHFxfU6m+0kbMZyB5y9g4PkLCG3dRsWLS6koKWXnokXs/OMfCQ4f7t5Ucg6Z+RMRn31fTjYLki5AVbnmmmv453/+533mrVq1iiVLlnDNNdcwZ86caG+iLcFgMLqLx+fzke5e+OXz+QiHnZsE/O53v2PIkCGsXbuWSCRCRkZGdPmvvvoKn8/H1q1biUQi+Fr5Y4y1K0lVKS4u5rHHHmtzmdZ2RTXU7ff7o3W3pF+/fsycOZPnn3/egqSXCA4ZzIBzzmHAOecQ/vZbKl96iT0lJXzzyCN88+CDBAYPJnf2bHLnFJNVUIAEbJOXDBbVKZKbm0tFRQUAJ554Ivfffz+VlZWAsxHftm0bmzdvJisri3PPPZcrrriC1atX77NsInbv3s3QoUPx+Xw8/PDD1NfXA87upPPPP59HH32UMWPGNNkl1dyMGTN45JFHAHj//fd5170r7LRp03j99dfZuNG5w391dXWTHs8TTzwR/behp9Ken2v79u3s2rULcI7XLF26lCOOOGK/1mF6hkD//vSbP5+D/vhHRr/xOsNuvZXMiRPZ9fe/88WC89lQNIPN115L5auvEqmrS3W5PYrFc4oMHDiQY489lnHjxjFv3jzOOeec6AY1JyeHv/zlL2zcuJErr7wSn89HMBjknnvuAeCiiy5i3rx5DB06tNXjJG255JJLmD9/Pk8++SSzZs0i272y+De/+Q1FRUUUFRWRn5/PlClTOPnkk2Neg3HxxRdz/vnnM2HCBPLz8yksLARg0KBBPPjgg5x99tnU1tYCcOONNzJ69GgAamtrmTp1KpFIJNprOeuss7jwwgu58847eeqpp1qse/PmzSxcuJAlS5awZcsWfvSjH1FfX08kEuGMM87glFNOafdnYnoGf24ufb97Cn2/ewqR6moqX3vNOVj/3PPsfupv+HJyyJk1i9w5xeRMn44vMzPVJXdrbV6QKCKjgHL3xoozcW5X8mdV3dUJ9XUIuyCxaxk5ciRlZWXk5eV16vva79xE6uqoeuMNKkpLqXzxJep37UIyM8kpKnJu1TLzOPzuWYCmY++19TegQEQOA+4DFgOPAie1upQxxnQxvrQ0cmfOJHfmTPT6MNUrVzojQLpjq0gwSPYxxzihcvwsAv37p7rkbiGeIIm49806DbhdVX8vIu8kuzATn6lTp0Z3HTV4+OGHGT9+fIe+zwsvvMBVV13VZNohhxzC008/vd/rajj7yphUkkCA7KOPJvvooxly7bXsXbMmeq1K5auvgt9PVuEUZ7Cu2bMJDBqU6pK7rHh2bb0N3A78Eviuqn4mIu+rarc5LcZ2bRmw37mJj6pS88E6d6z6Euo++wxEyDzqKHLnFNOnuJjg8OGpLrNTdOSurfOBnwA3uSFyCM7YIMYY0+OICJnjjiRz3JEM+tlPqdu4kT0lJVSULmXbzbew7eZbyDjySHLnzCF3TjHpNsbN/t39V0T6Aweq6rvJK6njWY/EgP3OTeLqPv+citJS9pSWUrPW2QymH34YucVzyD1xDumjR/eoW/R0WI9ERF7BGXQqAKwBtovIq6r6bwlXaYwx3UjawQczcOFCBi5cSGjLluhB+h333MOOu+8mePBB9HGvqs8YP75HhUpr4tm11VdV94jIQuABVb1ORLpVj8QYYzpacOhQBvzwPAb88DzCO3ZQ8eJLVJSWsvPBh9i56D4CQ4c6N5UsLiZz0iTE7091yUkTz5XtAREZCpwB/CPJ9fQaPfU28p09Hgk416WMHz+e/Px8Cgra7IUb0+ECeXn0P/MMDlr0J0a//hpDb/4tGWPGsOvxJ/j8vB+yYcZxbLnu11S+9joaCqW63A4XT5DcgDNc7iequlJEDgU2JLesnq+nBkki2jMeSYOXX36ZNWvWtDvEjOko/r596ff973Pg3Xcx+s03GP6728gqnMLuZ5/ly4UL+Xh6EZuvvoaKl14i0uzU/e6qzV1bqvok8KTn9afA/GQW1emeuxq+fq9j13nAeJh3c4uzbTySjhmPxJiuzJedTZ958+gzbx6RmhrnqvoXSqh46SV2P/MMvqwscmYeR25xMTkzZuBzb1PU3bTZIxGRESLytIhsE5GtIvI3ERnRGcX1ZDYeSceNRyIizJkzh8mTJ3Pvvfcm8FsxJnl8GRnkHn88w265mdGvLefARYvoc8opVL29gq8u/zc+PuZYvvyXS9n9v/9L/e7dqS53v8RzsP0BnFui/JP7+lx3WnFbC4rIXOAOwA8sUtWbm81fANwKfOVO+oOqLnLn3QKc7E7/D1V9wp0uwI1uPfXAPap6Zxw/R8ta6Tl0BhuPJLHxSF5//XWGDRvGtm3bKC4u5ogjjmDGjBlxfU7GpIKkpZEz/Vhyph/LAdf9ir2rV7OnpNS9B9iLEAiQPW0auXOKyT3hBAIDB6a65FbFEySDVPUBz+sHReRnbS0kIn7gLpzAKQdWishiVW2+E/wJVb202bInA5OAfCAdeFVEnlPVPcAC4EDgCFWNiMjgOH6GLs3GI0lsPJJhw4YBTg/rtNNOY8WKFRYkptsQv5+sKVPImjKFIddcTc177znXqpSU8vWvruPrX19P1uTJzgWQxbMJtrB3IJXiOdi+Q0TOFRG/+zgX2BnHcoXARlX9VFXrgMeB78VZ11jgVVUNq2oVsBaY6867GLhBVSMAqrotznV2KTYeSceMR1JVVRVdpqqqipKSEhvUynRb4vOROXEig6+4glEvPM8hzzxN3k9+Qv2ub9l6001snDmLz848k5333Ufdl1+mutyoeILkxzin/n4NbAFOx7ltSluGA96ftNyd1tx8EXlXRJ4SkQPdaWuBeSKSJSJ5wCycXgjAKOBMESkTkedE5PBYby4iF7ltyrZv3x5HuZ3LOx5JaWlpdDyS8ePHc/rpp1NRUcF7771HYWEh+fn53HTTTVx77bVA43gks2bNSqiGSy65hIceeohp06bx8ccfxxyP5LbbbmPRokWsX78+5jouvvhiKisrmTBhAv/5n/8ZczySCRMmMG3aND788MNVRH1CAAAa5UlEQVTocg3jkdxxxx387ne/A5zxSG699VaOOuooPvnkkxbr9h4j2bp1K9OnT2fixIkUFhZy8sknR3fpGdOdiQgZRxzBoMv+lUOffZZDlyxh0OWXQ7iebbf+F58Uz+HT75/G9rvvptb9wpayWvfnFinRhUR+pqq3t9Hmn4ATVXWh+/o8oFBV/9XTZiBQ6Y518hPgDFU93p33S5zjINuBbcAKVb1DRCqB61T1v0XkB8DlqtrqwQO7RUrXYuORGJOYuvKvqFhaSkVJKXvfeQdUSTv0UHes+mIyxo7tkKvq471FSnuH2o3n9ijlNPYiAEYAm70NVHWnqjacSP0nYLJn3k2qmq+qxYDQeO1KOc4YKQBP4wy0ZYwxvUbaiOEMXLCAkY8+wmGvvsKQX/07gSGD2bloEZvmn84ns4vZevMtVK9+B41Ekl5Pe4fajSfqVgKHu3cL/go4CzinyUpEhqrqFvflqcB6d7of6KeqO0VkAk5YlLjtngGOB+4HjgM+phez8UiM6d2Cgwcz4JxzGHDOOYS//ZbKl16ioqSUbx95hG8efJCRTz5J5vjkHjdsb5C0uT/MHQzrUpyr4v3A/ar6gYjcAJSp6mLgMhE5FQgD3+CckQUQBJa7XbM9wLmq2nDKzs3AIyJyOVAJLGznz9AjvP32253yPieeeCInnnhip7yXMaZ9Av3702/+fPrNn099RQVVr79Oxrgjk/++Lc0QkQpiB4YAmfGsXFWXAEuaTfuV5/k1wDUxlqvBOXMr1jp30Xh9iTHGmBj8ubn06aQTT1oMElXN7ZQKjDHGdGvtPdhujDHGABYkxhhjEmRBkiI99TbynT0eyUcffUR+fn700adPH26/vdVLnIwxHcyCJEV6apAkoj3jkXznO99hzZo10bsmZ2VlcdpppyWpQmNMLPGM2R7r7K3dQBnw/7njk3Rrt6y4hQ+/+bDthvvhiAFHcFXhVS3Ot/FIOn48khdffJFRo0Zx8MEHt/0LMsZ0mHh6JLcBV+LcJ2sEcAXOVeiP41wUaNrBxiPpuPFIGjz++OOt3vLeGJMc8VyQOFdVp3pe3ysib6nqDSLyi2QV1pla6zl0BhuPJLHxSBrWv3jxYn7729+2+dkYYzpWPEESEZEzgKfc16d75u3/HR/NPmw8ksTGIwF47rnnmDRpEkOGDGmxjTEmOeLZtfX/gPNw7sC7zX1+rohkApe2tqBpmY1H0jHjkTR47LHHbLeWMSnSZo/EPZj+3RZmv9ax5fQe3vFI5s2bFx2PBJwD5X/5y1/YuHEjV155JT6fj2AwyD333AM0jkcydOjQVo+TtOWSSy5h/vz5PPnkk8yaNSvmeCT5+flMmTKFk08+OeYt2C+++GLOP/98JkyYQH5+fszxSBpuKnnjjTcyevRooHE8kkgkEu21nHXWWVx44YXceeedPPXUU/u8V4PNmzezcOHC6O6t6upqSktL+eMf/9juz8IY035tjkciIiOA3wPH4uzKeg34qaqWJ7+8jmHjkXQtNh6JMd1DR45H8gCwGBiGc+bWs+40Y4wxJq6D7YNU1RscD4rIz5JVkNk/Nh6JMSbV4gmSHSJyLtBw+s3ZwM7klWT2h41HYoxJtXh2bf0YOAP4GtiCc/rv+cksyhhjTPfRZpCo6heqeqqqDlLVwar6feAHnVCbMcaYbqC9N238tw6twhhjTLfV3iBp+VJkY4wxvUp7gySuW6OIyFwR+UhENorI1THmLxCR7SKyxn0s9My7RUTedx9nxlj29yJS2c76U66n3ka+s8cjAbjjjjsYN24cRx55pI1FYkwKtBgkIlIhIntiPCpwrilplYj4gbuAecBY4GwRGRuj6ROqmu8+FrnLngxMAvKBqcCVItLHs+4CoN9+/JxdTk8NkkS0ZzyS999/nz/96U+sWLGCtWvX8o9//IMNGzYkqUJjTCwtnv6rqrkJrrsQ2NgwXomIPA58D4hnSzEWeFVVw0BYRNYCc4G/ugF1K3AO0CEjGH39m99Qu75jxyNJH3MEB/yi5Zsj23gkHTMeyfr165k2bRpZWVkAHHfccTz99NP8/Oc/34/fljEmEckcIXE48KXndbk7rbn5IvKuiDwlIge609YC80QkS0TygFlAw7xLgcWquiVZhXcGG4+kY8YjGTduHMuWLWPnzp1UV1ezZMkSvvzyS4wxnSeeCxLbK9YB+ebHVp4FHlPVWhH5CfAQcLyqlojIFOANYDvwJk7PZBjwT8DMNt9c5CLgIoCDDjqo1bat9Rw6g41H0v7xSMaMGcNVV11FcXExOTk5TJw4kUAgmf+tjTHNJfMvrpzGXgQ4oytu9jZQVe8V8n8CbvHMuwm4CUBEHgU2AEcBhwEb3TEsskRko6oe1vzNVfVe4F5wbtrYAT9P0th4JImNR3LBBRdwwQUXAPCLX/yCESNGtLg+Y0zHS+aurZXA4SJyiIikAWfh3PwxSkSGel6eCqx3p/tFZKD7fAIwAShR1f9T1QNUdaSqjgSqY4VId2DjkXTceCTbtm0D4IsvvuDvf/+7jUtiTCdLWo9EVcMicinwAuAH7lfVD0TkBqBMVRcDl4nIqUAY+AZY4C4eBJa731T3AOe6B957DBuPpOPGI5k/fz47d+4kGAxy11130b9//3Z/JsaY/dfmeCQ9gY1H0rXYeCTGdA8dOR6JMcYY0yI7vaWbs/FIjDGp1quDRFVbPWOoO7DxSOLTG3bhGpMqvXbXVkZGBjt37rQNTC+gquzcubPJ6c3GmI7Ta3skI0aMoLy8nO3bt6e6FNMJMjIy7PoSY5Kk1wZJMBjkkEMOSXUZxhjT7fXaXVvGGGM6hgWJMcaYhFiQGGOMSYgFiTHGmIRYkBhjjEmIBYkxxpiEWJAYY4xJiAWJMcaYhFiQGGOMSYgFiTHGmIRYkBhjjEmIBYkxxpiEWJAYY4xJiAWJMcaYhCQ1SERkroh8JCIbReTqGPMXiMh2EVnjPhZ65t0iIu+7jzM90x9x1/m+iNwvIsFk/gzGGGNal7QgERE/cBcwDxgLnC0iY2M0fUJV893HInfZk4FJQD4wFbhSRPq47R8BjgDGA5nAwhjrNMYY00mS2SMpBDaq6qeqWgc8DnwvzmXHAq+qalhVq4C1wFwAVV2iLmAFYMPeGWNMCiUzSIYDX3pel7vTmpsvIu+KyFMicqA7bS0wT0SyRCQPmAUc6F3I3aV1HvB8rDcXkYtEpExEymw4XWOMSZ5kBonEmKbNXj8LjFTVCcBS4CEAVS0BlgBvAI8BbwLhZsveDSxT1eWx3lxV71XVAlUtGDRoUPt/CmOMMa1KZpCU07QXMQLY7G2gqjtVtdZ9+SdgsmfeTe5xk2KcUNrQME9ErgMGAf+WpNqNMcbEKZlBshI4XEQOEZE04CxgsbeBiAz1vDwVWO9O94vIQPf5BGACUOK+XgicCJytqpEk1m+MMSYOgWStWFXDInIp8ALgB+5X1Q9E5AagTFUXA5eJyKk4u62+ARa4iweB5SICsAc4V1Ubdm39D/A58KY7/++qekOyfg5jjDGtE+fkp56toKBAy8rKUl2GMcZ0KyKySlUL2mpnV7YbY4xJiAWJMcaYhFiQGGOMSYgFiTHGmIRYkBhjjEmIBYkxxpiEWJAYY4xJiAWJMcaYhFiQGGOMSYgFiTHGmIRYkBhjjEmIBYkxxpiEWJAYY4xJiAWJMcaYhFiQGGOMSYgFiTHGmIRYkBhjjEmIBYkxxpiEWJAYY4xJSFKDRETmishHIrJRRK6OMX+BiGwXkTXuY6Fn3i0i8r77ONMz/RAReVtENojIEyKSlsyfwRhjTOuSFiQi4gfuAuYBY4GzRWRsjKZPqGq++1jkLnsyMAnIB6YCV4pIH7f9LcDvVPVw4FvggmT9DMYYY9qWzB5JIbBRVT9V1TrgceB7cS47FnhVVcOqWgWsBeaKiADHA0+57R4Cvt/BdRtjjNkPyQyS4cCXntfl7rTm5ovIuyLylIgc6E5bC8wTkSwRyQNmAQcCA4FdqhpuY53GGGM6STKDRGJM02avnwVGquoEYClODwNVLQGWAG8AjwFvAuE41+m8uchFIlImImXbt29v309gjDGmTckMknKcXkSDEcBmbwNV3amqte7LPwGTPfNuco+bFOMEyAZgB9BPRAItrdOz/L2qWqCqBYMGDeqQH8gYY8y+khkkK4HD3bOs0oCzgMXeBiIy1PPyVGC9O90vIgPd5xOACUCJqirwMnC6u8yPgP9N4s9gjDGmDYG2m7SPqoZF5FLgBcAP3K+qH4jIDUCZqi4GLhORU3F2W30DLHAXDwLLnWPr7AHO9RwXuQp4XERuBN4B7kvWz2CMMaZt4nzJ79kKCgq0rKws1WWY3koV6iqhajtU7YTqHe7zHVBbAb6A8/AHGp/7guDzu9ODnukBzzS/p207lhcfSKzDjsY4RGSVqha01S5pPRJjerS6KicIqnY0DYaq7VC90/PanVZfG3s94get79zaveIOIn9ju5baxgqsuJb3tos13fs+7Vk+aIGZZBYkxgCE9rYQBDGCoXoHhKpjryeQAdmDIDvP+XfwWMge6DzPcqd5X6dlOT2WSD1Ewu4j1Pi6PuROq3enN0yr97R150fbtrF8JAz14VaW9zxiLl8P4TqIVO//e2mkc3+vDcS3H0HUUujF6uV52sbqJTYJ2E5YPkW9TAsS0zOFaz1BsKNp72Cf1zudXU+x+NMagyErD/JGuyGR5wkGz+u07P3/QxZxNlb+XvDnGIk4PbC4Q8sbRJ6QjLV8k5Btafn9aevOr6tuYfnW6gql7jNuHkQLX4S8w5L6lr3gf67pEcJ1jT2DeIKhds8+q4gAIV+AcM4gQpkDCGUNINxvIqHM/oQz+xJK70M4oy+hjBxCadmE0rIJ+4OENEwoEiIcCROqDxFW599QJES47itCNZsIb/e0iYSaPG++nHd5gJxgDtnB7OgjJ8157Z2eE8whO63ptAx/BtLddtn4fIDP+fbc03l7mfGGVpNAjRV6nl5kvMtn9Gm71gRZkJj9Vh+pb9woRsL7bjSbbVBjbmDDtYT27iJcs4tQ7W7CtXsI1VYQrqskFKokHKomFNpLKLSXcH2NsxwQEiEsQkgghPvcHyTsCxAK+gn3DxAacAAhGUoYJaRKmAihSD0RvLtVKpxH9efQwl6q/SEIQV+QgC9A0B8kIM3+9QUI+oLRNhmBDHJ8OSjK3tBeyivLqQpVURmqpKquinD0JMWW+cUfDZmsYFaTsGkyzRNE2YHYQRXw2aagw/n8zoP0VFeSdPa/J4Xi2SC39jzeDXnMefWtzGtj+UgS93OLKkEggBAUH4F0H0Ffn+iGOOBPI+hPJxjIJBDIICOQSY4/0LgR92ysY23Ym8/fp613WoxAiNU+4AvgF3+H9Q5UlbpIHZV1lY3hEqpqEjT7THP/3V2zm82Vm6NtqsPxpWSGPyPaG8oKZLXcK/L0mJpMc//NDGR2v16SSZgFSSv+8ek/+Gz3Z61vbNvYILc2L5kbZIA0X9p+f0OObiAlSFAjBCJhgvUhAuE6gvV1BEI1BMM1BOv2EqyrJhCqIlhbSbC2igARggoBVYINgaAQTM8lkN6PYGZ/Apn9CGYNJJiZRyBrIMHsQQRyhhDMHkwg5wD8WQOR3nCsoBUiQro/nfTMdAZmDkxoXfWReqrD1XEHUXRaXSVbKrc4YRSqpiJUQTjSdi/JJz6yA9n77IbbJ4hitGkeSsHesPurh+jdf7FtWPLpEpZ/tTy6wW3XBrmrfEOORKBmV8tnJFXsgKotnuMQO1s+wyazf+OB5r4Ht3xGUvYgp20vD4ZU8vv85KblkpuWm/C66urrokFUFa5quccUapzX8Pzrqq+j86pD1WjsW+Q1keZLi9kzaumYUfMgagitzEAmPrEx/JLJLkhsRX2kHp/4umZXXRVqdscIhhgHnhv+bekbZUbf2GcgxXqdNaB3HCg1SRPRCHvDe9sOonBVk95TQ+/I26YuUtfm+wlCdjA7eszI2zOKazeep/eU5u9d4+jZBYkdwO/zd96bqTpXObd5RpLndUunGKb3gSy3Z9DvIBg+qZVgGAiB3vXHYVLLJ77oBjtRofpQy7vn2tiNt616W5OwiqeXFPQF9z2RoZVjRi31nrICWZ27fUkyC5JkUXWuft6fYGjp6ue0nMZg6DMchk5sPRiCGZ37sxqTIkF/kH7+fvTL6JfQelTV6SXtRxA1TNtevZ1NoU3RHlNNfU1c75kVyGpyNp03bNoKIu/zdH96yveaWJDsj7pqz+0wvNc0NH/tPsJ7Y68nkNl4PCFnCAw+sjEQoscXPEERzOzcn9OYXkZEyApmkRXMSnhdoUgouguuss45cy7e40k79+5sMi+eE3ICEmj15IaLJ17MkOwhCf9crdaQ1LV3d0t/DZ8tawyKUFXsdv70pgea874TIxg889MS79IbY7qmoC9I3/S+9E3vm9B6VJWa+pomYdPmiQ3utG9rvqW8opzKUCUXjL+gg36yllmQtCZS7xyIHjAq9hlJ0WDIsZvCGWM6lIiQGcgkM5BJXmZeqstplQVJa+b8R6orMMaYLs9OrjbGGJMQCxJjjDEJsSAxxhiTEAsSY4wxCbEgMcYYkxALEmOMMQmxIDHGGJMQCxJjjDEJ6RW3kReR7cDn7Vw8D9jRgeV0FKtr/1hd+8fq2j89ta6DVXVQW416RZAkQkTK4rkff2ezuvaP1bV/rK7909vrsl1bxhhjEmJBYowxJiEWJG27N9UFtMDq2j9W1/6xuvZPr67LjpEYY4xJiPVIjDHGJMSCBBCR+0Vkm4i838J8EZE7RWSjiLwrIpO6SF0zRWS3iKxxH7/qpLoOFJGXRWS9iHwgIj+N0abTP7M46+r0z0xEMkRkhYisdeu6PkabdBF5wv283haRkV2krgUist3zeS1Mdl2e9/aLyDsi8o8Y8zr984qzrpR8XiKySUTec9+zLMb85P49qmqvfwAzgEnA+y3MPwl4DhBgGvB2F6lrJvCPFHxeQ4FJ7vNc4GNgbKo/szjr6vTPzP0MctznQeBtYFqzNpcA/+M+Pwt4oovUtQD4Q2f/H3Pf+9+AR2P9vlLxecVZV0o+L2ATkNfK/KT+PVqPBFDVZcA3rTT5HvBndbwF9BORoV2grpRQ1S2qutp9XgGsB4Y3a9bpn1mcdXU69zOodF8G3Ufzg5PfAx5ynz8FnCCS3PGb46wrJURkBHAysKiFJp3+ecVZV1eV1L9HC5L4DAe+9LwupwtsoFxHu7smnhORIzv7zd1dCkfhfJv1Suln1kpdkILPzN0dsgbYBpSqaoufl6qGgd3AwC5QF8B8d3fIUyJyYLJrct0O/ByItDA/JZ9XHHVBaj4vBUpEZJWIXBRjflL/Hi1I4hPrm05X+Oa2GucWBhOB3wPPdOabi0gO8DfgZ6q6p/nsGIt0ymfWRl0p+cxUtV5V84ERQKGIjGvWJCWfVxx1PQuMVNUJwFIaewFJIyKnANtUdVVrzWJMS+rnFWddnf55uY5V1UnAPOBfRGRGs/lJ/bwsSOJTDni/WYwANqeolihV3dOwa0JVlwBBEcnrjPcWkSDOxvoRVf17jCYp+czaqiuVn5n7nruAV4C5zWZFPy8RCQB96cTdmi3Vpao7VbXWffknYHInlHMscKqIbAIeB44Xkb80a5OKz6vNulL0eaGqm91/twFPA4XNmiT179GCJD6LgR+6Zz5MA3ar6pZUFyUiBzTsFxaRQpzf585OeF8B7gPWq+ptLTTr9M8snrpS8ZmJyCAR6ec+zwRmAx82a7YY+JH7/HTgJXWPkqayrmb70U/FOe6UVKp6jaqOUNWROAfSX1LVc5s16/TPK566UvF5iUi2iOQ2PAfmAM3P9Ezq32Ogo1bUnYnIYzhn8+SJSDlwHc6BR1T1f4AlOGc9bASqgfO7SF2nAxeLSBjYC5yV7D8m17HAecB77v51gF8AB3lqS8VnFk9dqfjMhgIPiYgfJ7j+qqr/EJEbgDJVXYwTgA+LyEacb9ZnJbmmeOu6TEROBcJuXQs6oa6YusDnFU9dqfi8hgBPu9+PAsCjqvq8iPwEOufv0a5sN8YYkxDbtWWMMSYhFiTGGGMSYkFijDEmIRYkxhhjEmJBYowxJiEWJMYYYxJiQWJ6LRE5VUSu7sD1vSIiBTGmF4jIne7zBSLyhxaWr4w1vQPqWtJw4WErbVqqPV9ETkpGXabnsAsSTa/lXkC2uBPepwzYZ4yIzqKqiQRBPlCAc0GbMTFZj8T0SCIyUkQ+FJFFIvK+iDwiIrNF5HUR2SAihd7egYg8KM7AP2+IyKcicnob6/+5OAMJrRWRmz2z/kmcwaI+FpEit+1MiT0I0iEi8qaIrBSR/2jj/e52r5hGRJ4Wkfvd5xeIyI3u83Pd914jIn90r1hvGPQoz33+7+7nUioij4nIFS3VLiJpwA3Ame46z2z9Uze9lQWJ6ckOA+4AJgBHAOcA04ErcG6d0txQd/4pwM0x5gMgIvOA7wNT3bsI/6dndkBVC4Gf4dzSpjV3APeo6hTg6zbaLgOK3OfDgbHu8+nAchEZA5yJcxfYfKAe+H/N6i4A5uPcXv8HOD0Nrya1q2od8CucQaPyVfWJNmo0vZQFienJPlPV91Q1AnwAvOjeV+s9YGSM9s+oakRV1+Hcv6gls4EHVLUaQFW9d51tuOPwqhbew+tY4DH3+cNttF0OFInIWGAdsNW9QeDRwBvACTh3ml3p3mfsBODQZuuYDvyvqu51B/56ttn8/andmCg7RmJ6slrP84jndYTY//e97VsbbU9oeSyHhnXUt/AezcV1sztV/UpE+uPc5n0ZMAA4A6hU1Qr3jsYPqeo1bdTdmv2t3RjAeiTGtEcJ8GMRyQIQkQHtXM/rNN619v+11tD1Js5up2U4PZQr3H8BXgROF5HBDTWJyMHNln8N+K6IZIgz+NfJcbxnBZAbRzvTi1mQGLOfVPV5nLO9ytzdSFe0sUhLfoozmt1KnIGZ2rIc5zjGRpyRHge403B3x12LM9zqu0ApzjEfb90r3brX4uzGKsMZorY1LwNj7WC7aY3dRt6YXkREclS10u1NLQMuUtXVqa7LdG+2H9SY3uVe94B9Bs4xFQsRkzDrkRjTAhEZz75nU9Wq6tSe9J7GJMqCxBhjTELsYLsxxpiEWJAYY4xJiAWJMcaYhFiQGGOMSYgFiTHGmIT8/4p8WaAKcaNNAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xbc558d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_1.best_score_, gsearch2_1.best_params_))\n",
    "test_means = gsearch2_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch2_1.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(max_depth), len(min_child_weight))\n",
    "train_scores = np.array(train_means).reshape(len(max_depth), len(min_child_weight))\n",
    "\n",
    "for i, value in enumerate(max_depth):\n",
    "    pyplot.plot(min_child_weight, -test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'min_child_weight' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 进一精调max_depth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': range(4, 7), 'min_child_weight': [1]}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "max_depth = range(4,7)\n",
    "min_child_weight = [1]\n",
    "param_test2_1 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59333, std: 0.00314, params: {'max_depth': 4, 'min_child_weight': 1},\n",
       "  mean: -0.58874, std: 0.00413, params: {'max_depth': 5, 'min_child_weight': 1},\n",
       "  mean: -0.58927, std: 0.00345, params: {'max_depth': 6, 'min_child_weight': 1}],\n",
       " {'max_depth': 5, 'min_child_weight': 1},\n",
       " -0.5887351903284485)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=220,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_test2_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_1.grid_scores_, gsearch2_1.best_params_,     gsearch2_1.best_score_"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
